<?php
/**
 * @author yzs1013@163.com
 * filename Zqnews.class.php
 * 主要用予取得列表数据urls
 */
include_once(APP_PATH."/config.php");
include_once(APP_PATH . "/collect/Lib/ORG/Zqnews.class.php");
require_once(APP_PATH . "/collect/phpQuery.php");

class Zqindex extends Zqnews
{
    public $thisurl;
    public $resulty;
    public $msg;
    public $zq;
    private $bigtiltecss;
    public $firstinx;
    public $pageinfo;

    //列表 ★★★★★3.1处理列表数据
    public function __construct($url, $tempnr = '', $zq = "", $bug = 0)
    {
        $this->bug = $bug;
        if(!preg_match("/https?\:\/\//Uisx",$url)) $url=urldecode($url);
        $tempnr=$this->seturl($url, $tempnr, 6);
        if($tempnr=="kongHTML"){
            return "kongHTML";
        }
        if($this->gather_type==1&&!$GLOBALS['mycurl']->isNew){
            $this->bug=0;
            $this->getCacheTempnr($url);
            $this->pageinfo=$this->getdata();
            return $this;
        }
        $this->zq = ($zq != "") ? $zq : array();
        $this->getindex();
        if ($this->count() > 1 && $zq[0] == ""&&$_REQUEST['max']=="") {
            $this->msg = "";
            $this->msg .= $this->indexList();
        } else {
            $this->msg = "";
        }
    }

    public function getindex()
    {
        global $addreturn, $oldurl;
        $nn22 = array();
        if(is_array($GLOBALS['nn22'])&&!empty($GLOBALS['nn22'])){
            $nn22=$GLOBALS['nn22'];
            unset($GLOBALS['nn22']);
        }
        $rs = $this->rs;

        //列表 4.开始分析列表标题和链接
        //专用文件预处理
        $url = $this->url;
        $s_url_y = preg_replace("/[\?&]zq=[0-9]+/", "", $url);
        $this->getPageTitle();
        //列表 步骤 用preexe includetemp 来分析标题和链接 ==预先执行php
        if(empty($this->result)&&$this->gather_type==6) {
            if (trim($rs['preexe']) || $this->bug || is_array($this->jsdata)) {
                if(is_array($addreturn))extract($addreturn);
                if(is_array($addreturn['jsdata'])) extract($addreturn['jsdata']);
                if(is_array($this->jsdata)&&!empty($this->jsdata)){
                    $jsdata=$this->jsdata;
                }
                $oldurl = $url;
                ini_set('display_errors', 'off');
                $fn = fopen("includetemp.php", "w");
                fwrite($fn, "<" . "?php\r\n" . $rs['preexe'] . "\r\n?" . ">");
                fclose($fn);
                include("includetemp.php");
            }
        }

        $inifile = $_SERVER['DOCUMENT_ROOT'] . "/collect/Lib/CollcetLib/" . host($url) . ".inc.php";

        //echo $inifile;exit;
        if (is_file($inifile)) {
            include($inifile);
            $this->swfstr = trim(preg_replace("/\[([a-zA-Z0-9_]+)\]\[\/\\1\]/", "", $this->swfstr));
        }
        $this->clnn22($nn22);
     }

    protected function clnn22($nn22=array()){
         global $addreturn, $oldurl;
         $rs = $this->rs;
         if(!empty($GLOBALS['listdata'])){
             $nn22=$GLOBALS['listdata'];
             unset($GLOBALS['listdata']);
         }elseif(!empty($this->result)){
             $nn22=$this->result;
         }

         //列表 步骤 5.★★★★使用列表re分析列表标题和链接 列表页匹配正则: ====缓存分析==lists
         if (empty($nn22[0]) && $rs['hrefbtre'] != "") {
             $cc = preg_split("/\|\|\||$/m", trim($rs['hrefbtre']));
             $this->yuannr =($this->yuannr==""&&$this->ytempnr!="")? $this->ytempnr:$this->yuannr;
             $this->ytempnr =($this->ytempnr==""&&$this->yuannr!="")? $this->yuannr:$this->ytempnr;
             for ($i = 0; $i < count($cc); $i++) {
                 if (trim($cc[$i]) == "") continue;
                 $tempnr = $this->ytempnr;
                 if (preg_match("/\:to([a-z0-9_]+)(?:\:|\=|$)([^\r\n]*?)/Uimx", trim($cc[$i]), $mm5)) {
                     if ($addreturn[$mm5[1]] == "") {
                         $temp = preg_replace("/\:to([a-z0-9_]+)(?=\:|\=|$)([^\r\n]*?)/Uisx", "", trim($cc[$i]));
                         @preg_match_all("/" . trim(chop($temp)) . "/Uisx", $tempnr, $mm6);
                         if ($mm6[2][0] != "") {
                             $addreturn[$mm5[1]] = $mm6;
                         } elseif ($mm6[1][0] != null) {
                             $addreturn[$mm5[1]] = $mm6[1][0];
                             if ($mm5[1] == "jsdata") {
                                 $addreturn[$mm5[1]] = jsdata($mm6[1][0]);
                                 $this->logre("re", $cc[$i],$addreturn[$mm5[1]]);
                             }
                         } else {
                             $addreturn[$mm5[1]] = preg_replace("/@'(.*)'/", "\\1", $mm5[2]);
                             if ($mm5[1] == "baseurl") {
                                 $this->baseurl = $addreturn[$mm5[1]];
                             }
                         }
                     }
                 }


                 if (!preg_match("/\&\&\&/Uisx", $cc[$i])) {
                     if (preg_match("/\:nn([0-9]+)(?=$|\:)/Uisx", trim($cc[$i]), $mm4)) {
                         $cc[$i] = preg_replace("/\:nn[0-9]+(?=$|\:)/Uisx", "", trim($cc[$i]));
                         if (empty(${"nn" . $mm4[1]}) || empty(${"nn" . $mm4[1]}[0])) {
                             preg_match_all("/" . trim($cc[$i]) . "/Uisx", $tempnr, ${"nn" . $mm4[1]});
                             $a = 1;
                             $this->logre("re", $cc[$i],${"nn" . $mm4[1]});
                         }
                     } else {
                         $cc[$i] = trim($cc[$i]);
                         preg_match_all("/" . trim($cc[$i]) . "/Uisx", $tempnr, $nn);
                         ${"nn" . $mm4[1]} = trim($nn[1][0]);
                     }
                 } else {
                     $ddd = explode("&&&", $cc[$i]);
                     $tempnr = $this->ytempnr;

                     for ($d = 0; $d < count($ddd) - 1; $d++) {
                         preg_match_all("/" . trim($ddd[$d]) . "/Uisx", $tempnr, $nn);
                         if (trim($nn[1][0]) == "") {
                             $iscontinue = 0;
                             break;
                         } else {
                             $iscontinue = 1;
                             $tempnr = $nn[1][0];
                         }
                     }
                     if (!$iscontinue) continue;
                     if (preg_match("/\:nn([0-9]+)(?=\:|$)/Uisx", trim($ddd[count($ddd) - 1]), $mm3)) {
                         $ddd[count($ddd) - 1] = preg_replace("/\:nn[0-9]+(?=\:|$)/Uisx", "", trim($ddd[count($ddd) - 1]));
                         preg_match_all("/" . trim($ddd[count($ddd) - 1]) . "/Uisx", $tempnr, ${"tnn" . $mm3[1]});
                         if(is_array(${"tnn" . $mm3[1]}[0])&&!empty(${"tnn" . $mm3[1]}[0])) {
                             if (count(${"tnn" . $mm3[1]}[0]) > count(${"nn" . $mm3[1]}[0])) {
                                 ${"nn" . $mm3[1]} = ${"tnn" . $mm3[1]};
                                 $this->logre("re", $cc[$i]);
                             }
                         }
                         if (trim(${"tnn" . $mm3[1]}[1][0]) == "") {
                             continue;
                         }
                     } else {
                         ${"nn" . $mm3[1]} = ${"tnn" . $mm3[1]};
                     }
                 }
                 if (trim(${"tnn" . $mm3[1]}[1][0]) != "") {
                     break;
                 }
             }
             if (preg_match("/<(a|div)/Uisx", $nn22[1][0]) || preg_match("/<(a|div)/Uisx", $nn22[2][0])) {
                 preg_match_all("/<a[^<>]*href=\"(?<url>[^<>]*)\"[^<>]*>(?<title>.*)<\/a>/Uisx", $nn22[0][0], $nn22);
             }
         }

         //列表 6.★★★★使用列表pq分析列表标题和链接===clpq List== 列表页PQ:
         if (empty($nn22[0]) && $rs['hrefbtpq'] != "") {
             $cc = preg_split("/\|\|\||$/m", trim($rs['hrefbtpq']));
             $this->yuannr =($this->yuannr==""&&$this->ytempnr!="")? $this->ytempnr:$this->yuannr;
             $this->ytempnr =($this->ytempnr==""&&$this->yuannr!="")? $this->yuannr:$this->ytempnr;
             if(!$this->pqy) {
                 $this->ytempnr=pqhtml($this->ytempnr);
                 $this->pqy = phpQuery::newDocumentHTML(pqhtml($this->ytempnr));
             }
             $add = array();
             $quanwen = pq("")->htmlOuter();
             for ($i = 0; $i < count($cc); $i++) {
                 if (trim($cc[$i]) == "") continue;

                 $cc[$i] = trim($cc[$i]);
                 if (preg_match("/\:add/Uisx", trim($cc[$i]), $mm4)) {
                     $cc[$i] = preg_replace("/\:add/Uisx", "", trim($cc[$i]));
                     $cc_this=$cc[$i];
                     $nn = preg_split("/--/", preg_replace("/(\:add|\:nn[0-9]).*$/", "", $cc[$i]));
                     if (count($nn) > 0) {
                         $cc_this = preg_replace("/(\-\-|\:add|\:nn[0-9]).*$/", "", $cc[$i]);
                     }
                     $isadd = 1;
                 } else {
                     $isadd = 0;
                 }

                 //&&&代表分作几块来抓取
                 $hh = preg_split("/&&&/", $cc_this);
                 if ($hh[1] != "") {
                     $aSstrs = pq($hh[0]);
                     foreach ($aSstrs as $li2) {
                         $hh0_content = pq($li2)->htmlOuter();
                         for ($j = 1; $j < count($nn); $j++) {
                             if ($nn[$j] != "") {
                                 $pq2=phpQuery::newDocumentHTML(pqhtml($hh0_content));
                                 $replaces = pq($nn[$j],$pq2);
                                 for ($e = 0; $e < count($replaces); $e++) {
                                     $hh0_content = trim(str_replace(trim(pq($replaces->elements[$e])->htmlOuter()), "", $hh0_content));
                                 }
                             }
                         }
                         phpQuery::newDocument($hh0_content);
                         $a_count = count(pq($hh[1])->elements);
                         $add[] = array();
                         for ($a = 0; $a < $a_count; $a++) {
                             $title = trim(pq($hh[1])->eq($a)->text());
                             $title = ($title == "") ? pq($li2)->attr("alt") : $title;
                             $title = ($title == "") ? pq($li2)->attr("title") : $title;
                             $title = ($title == "") ? pq($li2)->find("img")->attr("alt") : $title;
                             $title = ($title == "") ? pq($li2)->find("img")->attr("title") : $title;
                             $url = pq($hh[1])->eq($a)->attr('href');
                             if ($url == "" && preg_match("/http[^<>'\"\s]+|[^<>'\"\s]+\.(htm|php)[^<>'\"\s]+/isx", pq($hh[1])->eq($a)->htmlOuter(), $nnuurr)) {
                                 $url = $nnuurr[0];
                             };
                             $nn22[0][] = $url;
                             $nn22['url'][] = $url;
                             $nn22[1][] = $title;
                             $nn22['title'][] = $title;
                             $add[count($add) - 1][] = array('title' => $title, 'url' => $url);
                             $this->logre("re", $cc[$i]);
                         }
                     }
                  } elseif(preg_match("/^([^,]*)\(([^,]*)\,([^,]*)(?:\,([^,]+))?(?:\,([^,]+))?(?:\,([^,]+))?(?:\,([^,]+))?\)/isx",$hh[0],$nn)) {
                     /*
                      * 括号方式：第一个是一条新闻的范围，第二个是范围里里的url，第三个是范围里的title,可以用|代表或者
                      * */
                     //参考:https://www.themalaysianinsight.com/
                     $hh0_content=$this->ytempnr;
                     $pq2=phpQuery::newDocumentHTML(pqhtml($hh0_content));
                     $aSstrs = pq($nn[1],$pq2);
                     $arr=array();
                     for ($e = 0; $e < count($aSstrs); $e++) {
                         $pq3=phpQuery::newDocumentHTML(pqhtml(pq($aSstrs->elements[$e])->htmlOuter()));
                         $nn[2]=str_replace("|",",",$nn[2]);
                         $nn[2]=preg_replace("/,\s*$/","",$nn[2]);
                         $url = pq($nn[2],$pq3)->attr('href');
                         if ($url == "" && preg_match("/https?\:[^<>'\"\s]+|[^<>'\"\s]+\.(htm|php)[^<>'\"\s]+/isx", pq("",$pq3)->htmlOuter(), $nnuurr)) {
                             $url = $nnuurr[0];
                         }
                         $nn[3]=str_replace("|",",",$nn[3]);
                         $nn[3]=preg_replace("/,\s*$/","",$nn[3]);
                         $title="";
                         $title = ($title == "") ? pq($nn[3],$pq3)->attr("alt") : $title;
                         $title = ($title == "") ? pq($nn[3],$pq3)->attr("title") : $title;
                         $title = ($title == "") ? trim(pq($nn[3],$pq3)->text()): $title;
                         $title = ($title == "") ? pq($nn[3],$pq3)->find("img")->attr("alt") : $title;
                         $title = ($title == "") ? pq($nn[3],$pq3)->find("img")->attr("title") : $title;
                         $nn[4]=str_replace("|",",",$nn[4]);
                         $nn[4]=preg_replace("/,\s*$/","",$nn[4]);
                         $thumb = trim(pq($nn[4],$pq3)->text());
                         $thumb = ($thumb == "") ? pq($nn[4],$pq3)->attr("src") : "";
                         $nn[5]=str_replace("|",",",$nn[5]);
                         $nn[5]=preg_replace("/,\s*$/","",$nn[5]);
                         $no = trim(pq($nn[5],$pq3)->text());
                         $no=($no!="")?$no:"";
                         $nn[6]=str_replace("|",",",$nn[6]);
                         $nn[6]=preg_replace("/,\s*$/","",$nn[6]);
                         $desc = trim(pq($nn[6],$pq3)->text());
                         $desc=($desc!="")?$desc:"";
                         $nn[7]=str_replace("|",",",$nn[7]);
                         $nn[7]=preg_replace("/,\s*$/","",$nn[7]);
                         $param = trim(pq($nn[7],$pq3)->text());
                         $param=($param!="")?$param:"";
                         $nn22[0][] = $url;
                         $nn22['url'][] = $url;
                         $nn22[1][] = $title;
                         $nn22['title'][] = $title;
                         $nn22[2][] = $title;
                         $nn22['thumb'][] = $thumb;
                         $nn22[3][] = $no;
                         $nn22['no'][] = $no;
                         $nn22[4][] = $desc;
                         $nn22['desc'][] = $desc;
                         $nn22[5][] = $param;
                         $nn22['param'][] = $param;
                         $newarr[] = array('title' => $title, 'url' => $url,'thumb'=>$thumb,"no"=>$no,"desc"=>$desc,"param"=>$param);
                     }
                     if($newarr) $add[]=$newarr;
                 } else {
                     //其它 （上面的&&&代表分作几块来抓取, >>代表删除  -> 代表属性)
                     $fen = preg_split("/>>/", $hh[0]);
                     $fenarr = pq($fen[0]);
                     $hh0_content=$this->ytempnr;
                     for ($j= 0; $j < count($fen); $j++) {

                         for ($jj = 1; $jj < count($nn); $jj++) {
                             if ($nn[$jj] != "") {
                                 $pq2=phpQuery::newDocumentHTML(trim($hh0_content));
                                 $replaces = pq($nn[$jj],$pq2);
                                 for ($e = 0; $e < count($replaces); $e++) {
                                     $hh0_content = trim(str_replace(trim(pq($replaces->elements[$e])->htmlOuter()), "", $hh0_content));
                                 }
                             }
                         }

                         if(count($fen)==1){
                             $pyfen = phpQuery::newDocument($hh0_content);
                         }else {
                             $pyfen = phpQuery::newDocument(pq($fenarr->elements[$j])->htmlOuter());
                         }
                         $aSstrs = pq($fen[$j],$pyfen);
                         $newarr = array();
                         foreach ($aSstrs as $li2) {
                             $title = trim(pq($li2)->text());
                             if(strlen($title)>50&&preg_match("/\n/",$title)) continue;
                             $title = ($title == "") ? pq($li2)->attr("alt") : $title;
                             $title = ($title == "") ? pq($li2)->attr("title") : $title;
                             $title = ($title == "") ? pq($li2)->find("img")->attr("alt") : $title;
                             $title = ($title == "") ? pq($li2)->find("img")->attr("title") : $title;
                             $url = pq($li2)->attr('href');
                             if ($url == "" && preg_match("/https?\:[^<>'\"\s]+|[^<>'\"\s]+\.(htm|php)[^<>'\"\s]+/isx", pq($li2)->htmlOuter(), $nnuurr)) {
                                 $url = $nnuurr[0];
                             };
                             $nn22[0][] = $url;
                             $nn22['url'][] = $url;
                             $nn22[1][] = $title;
                             $nn22['title'][] = $title;
                             $newarr[] = array('title' => $title, 'url' => $url);
                         }
                         if(!empty($newarr)){
                             $add[]=$newarr;
                         }
                     }
                     $nn22=$add;
                 }

                 if (count($nn22['url']) > 0||count($nn22[0][0]['url']) > 0) {
                     $this->logre("re", $cc[$i]);
                 }
                 if ($isadd) {
                     unset($nn22);
                 }
             }
         }

         //列表 步骤 用preexe includetemp 来分析标题和链接 列表页执行php
        //echo $rs['preexe'];exit;
         if (empty($nn22)&&trim($rs['preexe'])!=""&&(empty($this->result)||$this->bug||is_array($this->jsdata))) {
             if(is_array($addreturn)) extract($addreturn);
             if(is_array($addreturn['jsdata']))extract($addreturn['jsdata']);
             $oldurl = $url;
             ini_set('display_errors', 'off');
             $fn = fopen("includetemp.php", "w");
             fwrite($fn, "<" . "?php\r\n" . $rs['preexe'] . "\r\n?" . ">");
             fclose($fn);
             include("includetemp.php");
         }

         //## todo 6分析列表re公共正则
         //论坛公共正则1
         if (empty($nn22[0])) {
             preg_match_all("/<a[^<>]*href=\"(?<url>[^<>]*)\"[^<>]*class=\"s\s+xst\"[^<>]*>(?<title>.*)<\/a>/Uisx", $this->ytempnr, $nn22);
         }

         //论坛公共正则2
         if (empty($nn22[0])) {
             preg_match_all("/<a\s+onclick=\"atarget\(this\)\"\s+href=\"(?<url>[^<>]*)\">(?<title>[^<>]*)<\/a>/Uisx", $this->ytempnr, $nn22);
         }

         //列表 8.处理标题和链接数组 $nn
         $result = array();
         if (!empty($nn22[0]) && count($nn22[1]) > 0) {
             $result = array();
             if ($nn22['url'][0] != null && $nn22['url'][0] != "") {
                 for ($i = 0; $i < count($nn22[1]); $i++) {
                     $str = trim(strip_tags(trim($nn22['title'][$i])));
                     if ($str == "") {
                         $str = "(未知)";
                         if (preg_match("/alt=\"([^<>]*)\"/Uisx", $nn22['title'][$i], $nnss)) {
                             $str = $nnss[1];
                         }
                     }
                     $result[0][] = array("url" => $nn22['url'][$i], "title" => $str);
                 }
             } else if ($nn22['href'][0] != null && $nn22['href'][0] != "") {
                 for ($i = 0; $i < count($nn22[1]); $i++) {
                     $str = trim(strip_tags($nn22['title'][$i]));
                     $dateline=$nn22[$i]['datline'];
                     if ($str != "") $result[0][] = array("url" => $nn22['href'][$i], "title" => $str,"dateline" => $dateline);
                 }
             } else if ($nn22[0]['url'] != null && $nn22[0]['title'] != "") {
                 for ($i = 0; $i < count($nn22); $i++) {
                     $str = trim(strip_tags($nn22[$i]['title']));
                     $dateline=$nn22[$i]['datline'];
                     if ($str != "") $result[0][] = array("url" => $nn22[$i]['url'], "title" => $str,"dateline" => $dateline);
                 }
             } else {
                 for ($i = 0; $i < count($nn22[1]); $i++) {
                     if (preg_match("/^http\:\/\//i", $nn22[1][$i])) {
                         $result[0][] = array("url" => $nn22[1][$i], "title" => strip_tags($nn22[2][$i]));
                     } else if (preg_match("/^http\:\/\//i", $nn22[2][$i])) {
                         $result[0][] = array("url" => $nn22[2][$i], "title" => strip_tags($nn22[1][$i]));
                     } else {
                         preg_match_all("/<a[^<>]*href=[\"']??(?<href>[^<>\s\"\']*?)[\"']?[^<>]*>(?<title>.*)<\/a>/Uisx", $nn22[1][$i], $nn);
                         for ($j = 0; $j < count($nn[1]); $j++) {
                             $str = trim(strip_tags($nn[2][$j]));
                             if ($str == "") {
                                 preg_match("/(?:alt|title)=\s*\"(.*)\"/Uisx", $nn22[0][$i], $nn33);
                                 $str = $nn33[1];
                             }
                             if (strlen($str) > 12) {
                                 $result[0][] = array("url" => $nn[1][$j], "title" => $str);
                             }
                         }
                     }
                 }
             }
         }

         if (!empty($result) && $this->jslielist($result)) {
             $this->resulty['url'] = $nn22['url'];
             $this->resulty['title'] = $nn22['title'];
         }

         //列表 9.自动分析标题和链接数组
         if (!$this->jslielist($result)) {
             $thistemp = preg_replace("/\s(target|on[a-z]+|span|border)=\"[^<>\"]+\"/Uisx", "", $this->yuannr);
             $thistemp = preg_replace("/<\/?(link|meta)[^<>]>/Uisx", "", $thistemp);
             $thistemp = preg_replace("/border=[0-9]+/isx", "", $thistemp);
             $thistemp = preg_replace("/<(script|style)[^<>]*>.*<\/\\1>/Uisx", "", $thistemp);
             //$thistemp = preg_replace("/<div[^<>]*>(?:\s*<div[^<>]*>)?(<\/?(a|span|p|br|embed|object)[^<>]*>|\s|\&nbsp\;)*(<img[^<>]*>)(<" . "?\/?(a|span|p|br|embed|object)[^<>]*>|([^\<\>a-z0-9_\s]|\&nbsp\;|\s))*<\/div>(\s*<\/div>)??/Uuisx", "\\3\\4", $thistemp);

             $thistemp_y = $thistemp;
             $n = 0;
             $result = array();
             do {
                 preg_match_all("/<(div)[^<>]*>((?:.(?!\<\\1))*)<\/\\1>/Uisx", $thistemp, $nn);
                 for ($i = 0; $i < count($nn[0]); $i++) {
                     $ss = array();
                     preg_match_all("/<a\s[^<>]*href=([^\s\>]*?)[^<>]*>(.+)<\/a>/Uisx", $nn[0][$i], $ss);
                     if (count($ss[0]) >= 5) {
                         if (strlen($ss[2][$i]) > 10) {
                             $result[] = $this->tohb($ss);
                         }
                     }
                     $thistemp = str_replace($nn[0][$i], "", $thistemp);
                 }
                 $n++;
             } while (count($nn[0]) > 0 && $n < 20 && count($nn[0]) > 0);
             if (count($result) < 1) {
                 $thistemp = $thistemp_y;
                 do {
                     preg_match_all("/<(table)[^<>]*>((?:.(?!\<\\1))*)<\/\\1>/Uisx", $thistemp, $nn);
                     for ($i = 0; $i < count($nn[0]); $i++) {
                         $ss = array();
                         preg_match_all("/<a\s[^<>]*href=([^\s\>]*?)[^<>]*>(.+)<\/a>/Uisx", $nn[0][$i], $ss);
                         if (count($ss[0]) >= 5) {
                             if (strlen($ss[2][$i]) > 10) {
                                 $result[] = $this->tohb($ss);
                             }
                         }
                         $thistemp = str_replace($nn[0][$i], "", $thistemp);
                     }
                     $n++;
                 } while (count($nn[0]) > 0 && $n < 20 && count($nn[0]) > 0);
             }
             if (count($result) < 1) {
                 $thistemp = $thistemp_y;
                 do {
                     preg_match_all("/<(ul)[^<>]*>((?:.(?!\<\\1))*)<\/\\1>/Uisx", $thistemp, $nn);
                     for ($i = 0; $i < count($nn[0]); $i++) {
                         $ss = array();
                         preg_match_all("/<a\s[^<>]*href=([^\s\>]*?)[^<>]*>(.+)<\/a>/Uisx", $nn[0][$i], $ss);
                         if (count($ss[0]) >= 5) {
                             if (strlen($ss[2][$i]) > 10) {
                                 $result[] = $this->tohb($ss);
                             }
                         }
                         $thistemp = str_replace($nn[0][$i], "", $thistemp);
                     }
                     $n++;
                 } while (count($nn[0]) > 0 && $n < 20 && count($nn[0]) > 0);
             }
             //通过条数和每条长度、样式达到标准找到区域
             if (!$this->jslielist($result)) {
                 $rearr = array("div", "table", "ul");
                 foreach ($rearr as $k => $v) {
                     $thistemp = $thistemp_y;
                     do {
                         preg_match_all("/<" . $v . "[^<>]*>([\s\S]*)<\/" . $v . ">/Uisx", $thistemp, $nn);
                         for ($i = 0; $i < count($nn[0]); $i++) {
                             $result_temp = array();
                             $ss = array();
                             preg_match_all("/<a\s[^<>]*href=([^\s\>]*?)[^<>]*>(.+)<\/a>/Uisx", $nn[0][$i], $ss);
                             if (count($ss[2]) >= 5) {
                                 for ($j = 0; $j < count($ss[2]); $j++) {
                                     if (strlen(strip_tags($ss[2][$j])) > 20) {
                                         $result_temp[] = $this->tohb2($ss, $j);
                                     } elseif (preg_match("/详细|详情|点击/Uisx", strip_tags($ss[2][$j]))) {
                                         continue;
                                     } else {
                                         break;
                                     }
                                 }
                             }
                             $thistemp = str_replace($nn[0][$i], "", $thistemp);
                             if (count($result_temp) >= 5) $result[] = $result_temp;
                         }
                         $n++;
                     } while (count($nn[0]) > 0 && $n < 50 && count($nn[0]) > 0);
                 }

                 //通过本页样式找出
                 $thistemp = $thistemp_y;
                 $this->formNrFindList($thistemp, $result);

                 //通过样式文件找出
                 if(count($add)<1&&count($result)<1&&!in_array("14",explode(",",$this->rs['listzq']))) {
                     preg_match_all("/<link[^<>]*href=([^<>\s]*?\.css)[^<>]*>/Uisx", $this->yuannr, $nn);
                     for ($i = 0; $i < count($nn[1]); $i++) {
                         if (preg_match("/(mobile|pad)[\/_\.]/i", $nn[1][$i])) continue;
                         $nn[1][$i] = preg_replace("/\"|'|\r|\n/", "", $nn[1][$i]);
                         $cssurl = quanurl($nn[1][$i], $this->url);
                         if (urlexists($cssurl)) {
                             $opts = array(
                                 "http" => array(
                                     "method" => "GET",
                                     "timeout" => 2,
                                     "header" => "Connection:keep-alive;\r\nContent-type:application/x-www-form-urlencoded;charset=UTF-8;\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36\r\n",
                                 ),
                                 'ssl' => array(
                                     "verify_peer" => false,
                                     "verify_peer_name" => false,
                                 )
                             );
                             $context = stream_context_create($opts);
                             $cssfilenr = file_get_contents($cssurl, false, $context);
                             $this->formNrFindList($cssfilenr, $result);
                         }
                     }
                 }

             }
         }

         if (is_array($add)&&count($add) > 0) {
             $result = array_merge($result, $add);
             unset($add);
         }

         //列表 10.过滤垃圾列表
         $cc = count($result);
         for ($i = 0; $i < $cc; $i++) {
             $no = 0;
             $yes = 0;
             if (count($result[$i]) == 0){
                 unset($result[$i]);
             }
             foreach ($result[$i] as $j=>$v) {
                 if (preg_match("/^(首页|手机版|标签|管理中心)$/",$v['title'])) {
                     unset($result[$i][$j]);
                     break;
                 }
                 if (trim($result[$i][$j]['title']) != "(未知)" && trim($result[$i][$j]['title']) != "" && strlen(trim($result[$i][$j]['title'])) <= 12) {
                     //unset($result[$i][$j]);
                     $no++;
                 }
                 if (strlen(trim($result[$i][$j]['title'])) >= 18 || trim($result[$i][$j]['title']) != "(未知)" || trim($result[$i][$j]['title']) != "") {
                     $yes++;
                 }
             }
             if ($no >= 1 && $yes < 3) {
                 unset($result[$i]);
                 break;
             }

         }
         foreach ($result as $k => $a) {
             $this->firstinx = $k;
             break;
         }

         //===================列表处理result
        if($rs['listresultphp']) {
            ini_set('display_errors', 'off');
            $fn = fopen("includetemp5.php", "w");
            fwrite($fn, "<" . "?php\r\n" . $rs['listresultphp'] . "\r\n?" . ">");
            fclose($fn);
            include("includetemp5.php");
        }

         $result=object2array($result);
         $this->result = self::quanlj($result);
         $result = $this->result;
         $pageinfo=json_decode($this->pageinfo);
         $this->finish("getindex",$this->result);
         //yzs ==nn== ==result
         return $result;
    }

    protected function tohb($nn)
    {
        $arr = $oldurlarr = array();
        for ($i = 0; $i < count($nn[0]); $i++) {
            $url = preg_replace("/'|\"/", "", $nn[1][$i]);
            if (!in_array($nn[1][$i], $oldurlarr) && !preg_match("/javascript/", $nn[1][$i]) && !preg_match("/xe60d;|<img/", $nn[2][$i])) {
                $arr[] = array("url" => $url, "title" => trim(strip_tags($nn[2][$i])));
                $oldurlarr[] = $url;
            }
        }
        return $arr;
    }

    protected function tohb2($nn, $i)
    {
        $arr = array("url" => preg_replace("/'|\"/", "", $nn[1][$i]), "title" => trim(strip_tags($nn[2][$i])));
        return $arr;
    }

    protected function formNrFindList($thistemp, &$result)
    {
        preg_match_all("/<style>(.*)<\/style>/Uisx", $thistemp, $cssnn);
        if (count($cssnn[1]) > 0 || preg_match("/<head>/Uisx", $thistemp)) {
            $thistemp = join("\r\n", $cssnn[1]);
        }
        preg_match_all("/(\.(?:[^<>\s\{\}]+)|h[0-9]|(?<=\}|\r|\n)[^\{\}]*\b(?:a|em))\s*\{[^\r\n\{\}]*font(?:\-size)?(?:\:(?:2[0-9]|18|19))px[^\r\n\{\}]*\}/Uisx", $thistemp, $nn);
        if (count($nn[1]) > 0) {
            for ($t = 0; $t < count($nn[1]); $t++) {
                if ($this->bigtiltecss != null && in_array($nn[1][$t], $this->bigtiltecss)) continue;
                $this->bigtiltecss[] = $nn[1][$t];
                if (@preg_match("/h([0-9]\s*$/i", $nn[1][$t], $tts)) {
                    preg_match_all("/<(" . $nn[1][$t] . ")[^<>]*>.*<\/\\1>/Uisx", $this->yuannr, $nn22);
                    $tiaos = 3;
                } elseif (preg_match("/\b(a|em)\bs*$/i", $nn[1][$t], $tts)) {
                    $nn[1][$t] = trim($nn[1][$t]);
                    $prere = preg_replace("/\#([^\s]+)/", "id=(?<!\"|')[^\s]+\\1", $nn[1][$t]);
                    $prere = preg_replace("/\.([^\s]+)/", "class=(?<!\"|')[^\s]+\\1", $prere);
                    $prere = preg_replace("/\s+/", ".*", $prere);
                    $prere = preg_replace("/\.\*\b" . $tts[1] . "\b/", "", $prere);
                    @preg_match("/" . $prere . ".*?<\/" . $tts[1] . ">/isx", $this->yuannr, $area);
                    @preg_match_all("/<(?:" . $tts[1] . "\b)[^<>]*>.*<\/" . $tts[1] . ">/Uisx", $area[0], $nn22);
                    $tiaos = 5;
                } else {
                    if (substr($nn[1][$t], 0, 1) == ".") {
                        $class = "class";
                    }
                    if (substr($nn[1][$t], 0, 1) == "#") {
                        $class = "id";
                    }
                    $nn[1][$t] = preg_replace("/^(\.|\#)/", "", $nn[1][$t]);
                    preg_match_all("/<(td|li|ul|div|a|p)[^<>]*" . $class . "=\"[<>]*\b" . $nn[1][$t] . "\b[<>]*\"[^<>]*>.*<\/\\1>/Uisx", $this->yuannr, $nn22);
                    $tiaos = 5;
                }
                $result_temp = $oldarr = array();
                if (count($nn22[0]) >= $tiaos) {
                    for ($i = 0; $i < count($nn22[0]); $i++) {
                        preg_match("/<a\s[^<>]*href=([^\>\s]*?)[^<>]*>(.+)<\/a>/Uisx", $nn22[0][$i], $ss);
                        $title = trim(strip_tags($ss[2]));
                        if (preg_match("/<img/i", $title) || $title == "") {
                            preg_match("/(?:alt|title)=([^<>]+)/", $ss[0], $tt);
                            $title = preg_replace("/\"|'/", "", $tt[1]);
                        };
                        $url = preg_replace("/'|\"|\r|\n/", "", $ss[1]);
                        if (strlen($title) > 10 && !in_array($url, $oldarr)) {
                            $result_temp[] = array("url" => $url, "title" => $title);
                            $oldarr[] = $url;
                        }
                    }
                }
                if (!empty($result_temp)) $result[] = $result_temp;
            }
        }
    }

    protected function jslielist($result)
    {
        if (empty($result) || count($result) < 1) return false;
        $hasOneGt20 = 0;
        for ($i = 0; $i < count($result); $i++) {
            $oldurls = array();
            foreach ($result[$i] as $k => $v) {
                if (in_array($v['url'], $oldurls) || $v['url'] == "#") {
                    continue;
                }
                if (strlen($v['title']) > 20 || $v['title'] == "(未知)") {
                    $hasOneGt20 = 1;
                }
                $oldurls[] = $v['url'];
            }
        }
        if ($hasOneGt20 == 0) return false;
        return true;
    }

    protected function quanlj($arr)
    {
        foreach ($arr as $key => $a) {
            $harr = array();
            foreach ($arr[$key] as $k => $v) {
                $v['url'] = preg_replace("/^(\s|\'|\")+|(\s|\'|\")+$/", "", $v['url']);
                $v['url'] = quanurl($v['url'], $this->url, $this->baseurl);
                if (!in_array($v['url'], $harr)) {
                    $arr[$key][$k]['url'] = $v['url'];
                    $harr[] = $v['url'];
                } else {
                    //unset($arr[$key][$k]);
                }
            }
            $arr[$key] = self::rekey($arr[$key]);
        }
        return $arr;
    }

    protected function themax()
    {
        $max = array();
        $temp=array();
        foreach ($this->result as $k => $v) {
            if (count($v) > count($temp)) {
                $max = $v;
            }
        }
        return $max;
    }

    function theindex($index = "max")
    {
        if ($index[0] > 0) {
            $firstinx = $index[0];
        } elseif ($index == "max") {
            return self::themax();
        } else {
            foreach ($this->result as $k => $a) {
                $firstinx = $k;
                $this->firstinx = $k;
                break;
            }
        }

        if ($this->result[$firstinx] != null) {
            $arr = $this->result[$firstinx];
        } elseif(!empty($this->result[0])) {
            $arr = $this->result[0];
        }else{
            $arr=array();
        }
        return $arr;
    }

    protected function count()
    {
        return count($this->result);
    }

    function indexList()
    {
        $str = "";
        $arr = $this->result;
        foreach ($arr as $i => $a) {
            $wen = preg_match("/\?/", $this->url) ? "&" : "?";
            $str .= "\r\n<p>提示新网址: <a href=\"#\" onclick=\"fz(this)\">" . trim($this->url) . $wen . "zq=" . $i . "</a>，抓取以下信息： <br>\r\n";
            for ($j = 0; $j < count($arr[$i]); $j++) {
                $str .= $arr[$i][$j]['title'] . "\r\n";
            }
        }
        $str = ($str == "") ? "（没有找到数据!）" : $str;
        return $str;
    }

    function gettitle($url)
    {
        if ($url != "") {
            preg_match("/" . addsla($url) . "[^<>]*>(.*)<\/a>/", $this->nr, $nn);
            $title = trim(strip_tags($nn[1]));
            $title=preg_replace("/\<\!\-\-[^\<\>]*\-\-\>/Uisx","",$title);
        }
        return $title;
    }

    protected function rekey($arr)
    {
        $newarr = array();
        foreach ($arr as $k => $v) {
            $newarr[] = $v;
        }
        return $newarr;
    }

    function listToMessage($d){
        global $pageinfo,$api,$wapi;
        if(is_string($d[0])){
            $d[0]=json_decode($d[0],1);
        }
        $pageinfo['subject'] = $d[0]['subject'] . " 等多篇文章";
        $pageinfo['url'] = $this->url;
        $pageinfo['lists'] = $d;
        $pageinfo['listmax']=0;
        if (!empty($d)) {
            if ($api || $wapi) {
                $pageinfo['message'] = print_r($d, 1);
            } else {
                //列表 12.把最终数据完整打包
                foreach ($d as $k => $v) {
                    if(is_string($d[$k])){
                        $d[$k]=json_decode($d[$k],1);
                    }
                    if (is_array($d[$k]) && $d[$k]['url']){
                        $title=($d[$k]['subject']==""&&$d[$k]['title']!="")?$d[$k]['title']:$d[$k]['subject'];
                        $pageinfo['message'] .= "\r\n\r\n=================={$d[$k]['url']}=====================\r\n" . $title . "\r\n" . $d[$k]['message'];
                        $this->addnrdata($d[$k]['url'],$d);
                    }
                }
            }
        } else {
            $pageinfo['subject'] = "";
            $pageinfo['message'] = "";
        }
        return $d;
    }

    function toendjson(){
        global $pageinfo,$api,$wapi,$s_url_y,$mllb;
        $d=$this->zqlistnr($this->result,getlisturlid($this->url));
        if($pageinfo['message']==null){
            $mllb = new Zqindex($s_url_y);
            $mllb->listToMessage($d);
        }
        $pageinfo['lists']=$this->result;
        $pageinfojson=json_encode($pageinfo);
        //收集标题和列表完成后，保存一下pageinfo入库，开始进入抓取一篇篇文章
        $this->savePageinfo($this->url,$pageinfojson,"","","",array("gather_type"=>6));
        $this->savePageinfo($_REQUEST['s_url'],$pageinfojson,"","","",array("gather_type"=>6));
        endjson($pageinfo);
    }

    function zqlistnr($urls,$listid){
        global $listinfo,$jumpcacheids,$addreturn,$s_url;
        $this->thisurl=$this->url;
        $s_url=($s_url=="")?$this->url:$s_url;
        $listinfo['urls_info']['result'][$s_url]['count']=count($urls);
        $nozqnr=$_REQUEST['nozqnr'];
        $d=array();
        if(empty($urls)){
            $SQL = "select id,tempnr,pageinfo,url,title,dateline from `cache` where listurlid='$listid' and not (pageinfo is null or pageinfo='') order by id desc";
            $urls = query($SQL, 1);
            $isdofinished=true;
        }else{
            $isdofinished=false;
        }
        if ($this->listnogengxin) {
            $d = json_decode(trim($this->pageinfo), 1);
            $listinfo['urls_info']['result'][$s_url]['new']=0;
            $this->listPagesUrlsInsertToCacheByListId($d,$listid);
        } else {
            //yzs //列表 新旧列表
            $longnew=array();
            $localnew=array();
            $listurl=$this->getListIdUrl($listid);
            $status=$this->waitAndFinshed($listid,$urls,$listurl);
            $waitData=$status['wait'];
            $hasTempNrData=$status['hastempnr'];
            $finshedData=$status['finshed'];
            header("list-status:".count($urls)." ".count($finshedData)." ".count($hasTempNrData)." ".count($waitData)." {$listid}");
            $this->oldlistisfinshed=empty($waitData);
            $this->listPagesUrlsInsertToCacheByListId($urls,$listid);
            $this->do_array_merge($urls,$waitData,$finshedData,$longnew,$localnew);
            $listinfo['urls_info']['result'][$s_url]['new']=count($urls);
            $listinfo['urls_info'][$s_url]['longnew']=$longnew;
            $listinfo['urls_info'][$s_url]['localnew']=$localnew;
//            if ($nozqnr == 1) {
//                $newarr = array();
//                for ($i = 0; $i < count($urls); $i++) {
//                    if (preg_match("/(pdf|swf)$/", $urls[$i][url])) continue;
//                    $id = $this->t->sqlvalue("select id from `cache` where url='{$urls[$i][url]}'");
//                    if ($id <= 1 || $id == 0 || $this->bug == 1) {
//                        $newarr[] = $urls[$i][url];
//                    }
//                }
//                $newarr = array_unique($newarr);
//                $contents = multiDownload($newarr);
//                unset($newarr);
//                if (!empty($contents)) {
//                    foreach ($contents as $url => $content) {
//                        $this->saveCacheTempnr($url, $content);
//                    }
//                }
//                //启动多线程抓取网页
//                if (class_exists("Thread")) {
//                    $newArr = array();
//                    foreach ($urls as $k => $v) {
//                        if ($this->isHasPageInfo($v['url'])) continue;
//                        $newArr[] = $v;
//                    }
//                    if (count($newArr) > 0) $threadDataArray = createThreadMain($newArr, $this);
//                    unset($newArr);
//                }
//            }
            if(empty($waitData)&&$this->listisold&&$this->pageinfo!=""&&!$this->bug){
                echo $this->pageinfo;
                exit;
            }else {
                $yong = new CollectTongyong();

                //##todo 6最终列表新闻 ==urls==
                //列表 步骤9 获得列表标题和url数据后，开始一一一处理文章url
                if($urls[0][0]['url']!=""||$urls[0][0]['title']!=""){
                    $urls=$urls[0];
                }

                if($_REQUEST['bug']&&count($waitData)==0&&!$isdofinished){
                    for($i=0;$i<count($urls);$i++) {
                        query("update cache set pageinfo=(null),tempnr=(null),message=(null) where url='{$urls[$i]['url']}' and listurlid='{$listid}'");
                    }
                }


                for ($i8 = 0; $i8 < count($urls); $i8++) {
                    if (preg_match("/\.(pdf|swf|png|jpg|jpeg|gif|mp3|mp4|flv|mpeg|zip|rar)$/", $urls[$i8]['url'])) continue;

                    //echo $urls[$i][url]."\r\n";
                    try {
                        if ($urls[$i8]['url']) {
                            if ($nozqnr == 1) {
                                $d[] = array(
                                    "url" => $urls[$i8]['url'],
                                    "subject" => $urls[$i8]['title'],
                                );
                            } else {
                                $wbug = isset($_REQUEST['fromcacheid']) ? 0 : $this->bug;
                                $GLOBALS['guolvsamemaxstr']="";
                                unset($addreturn['jsdata']);
                                //last($urls[$i8]['url'], 1,$urls[$i8]['url'],$urls);
                                if ($_REQUEST['bug']) {
                                    $wbug = 1;
                                } elseif (!$GLOBALS['mycurl']->isNew) {
                                    $wbug = 0;
                                    $_REQUEST['fromcacheid'] = 1;
                                }

                                //列表 //yzs //内容 11.分页处理开始，进入篇篇文章url
                                /**
                                 * 1.不管列表有没有更新，没有抓取完的文章，要继续抓取,怎么判断有没有抓取完？
                                 * 2.如果bug=1,要重新抓取文章。
                                 * 3.如果有新的文章，要抓取新的文章。
                                 * 4.如果没有新更新，又没有旧的需要更新，请不可经过这一步，直接取列表页的pageinfo
                                 */
                                //列表 步骤10 列表开始处理新一篇内容
                                jsstart("onepage");
                                //内容 步骤14 列表或分页里的下一篇内容开始 getPageContent ==d[]==
                                $GLOBALS['debugger']['result_index']=$i8;
                                $GLOBALS['debugger']['result_url']=$urls[$i8]['url'];
                                $GLOBALS['debugger']['result_url_count']=count($urls);
                                //$tishi=$GLOBALS['debugger']['curr'][count($GLOBALS['debugger']['curr'])-1]['list_result'];
                                //$tishi=$urls[$i8]['url'];
                                if($finshedData[$urls[$i8]['url']]['pageinfo']&&$isdofinished){
                                    $d[] =json_decode($finshedData[$urls[$i8]['url']]['pageinfo'],1);
                                }else {
                                    fopenwtemp("列表","{$urls[$i8]['url']}",$this->url,0,array());
                                    $d[] = $yong->getPageContent($urls[$i8]['url'], 1, $wbug, "", $urls[$i8]['title'], $urls[$i8]['dateline'],$hasTempNrData[$urls[$i8]['url']]['tempnr']);
                                }

                                jsstop("onepage");
                                if (js("onepage") > $GLOBALS['onepage']) {//每篇文章最多抓5秒钟时间
                                    if (js("zhongsj") > $GLOBALS['zhongsj']) {//每次抓取最多抓30秒钟时间
                                        header("break_url:{$urls[$i8]['url']}");
                                        break;
                                    }else{
                                        continue;
                                    }
                                }
                                if (js("zhongsj") > $GLOBALS['zhongsj']) {//每次抓取最多抓30秒钟时间
                                    header("break_url:{$urls[$i8]['url']}");
                                    break;
                                }
                            }
                        }
                    } catch (Exception $e) {
                        echo $e->getMessage();
                    }
                }
            }
        }
        //内容 步骤15 将所有列表文章打包，准备输出 listToMessage
        //列表 步骤10 将所有列表文章打包，准备进行输出 listToMessage
        $this->result=$d;
        return $d;
    }

    //将找到的results的标题和链接入库
    function listPagesUrlsInsertToCacheByListId($d,$listid){
        if(is_array($d[0][0])){
            for ($i = 0; $i < count($d[0]); $i++) {
                listPagesUrlsInsertToCacheByListId($d[0][$i],$listid);
            }
        }else {
            for ($i = 0; $i < count($d); $i++) {
                $url = $d[$i]['url'];
                $title = $d[$i]['title'];
                $dateline = $d[$i]['dateline'];
                if ($url != "" && $listid != "") {
                    alistPagesUrlsInsertToCacheByListId($url, $listid, $title, $dateline);
                }
            }
        }
    }

    function do_array_merge($urls,$waitData,$finshedData,&$longnew,&$localnew){
        global $jumpcacheids;
        for($i=0;$i<count($waitData);$i++){
            if(in_array($waitData[$i]['title'],$finshedData)){
                $longnew[]=$waitData[$i];
            }
            $urlcode=urlcode($waitData[$i]['url'],$waitData[$i]['title'],$waitData[$i]['dateline']);
            $waitData[$i]['cacheid']=getone("select id from `cache` WHERE urlcode='{$urlcode}'");
            if($waitData[$i]['cacheid']!=null&&$jumpcacheids!=null&&!in_array($waitData[$i]['cacheid'],$jumpcacheids)){
                $newnew[]=$waitData[$i];
                $localnew[]=$waitData[$i];
            }
        }
        if(!empty($jumpcacheids)){
            return $localnew;
        }else {
            return $waitData;
        }
    }

    function getListIdUrl($id){
        $url=getOne("select url from lists_url where id='$id'");
        return $url;
    }

    function reultMergeCachedId(){
        $this->result;
        return  $this->result;
    }
}

//$a=new Zqindex("http://pypt.rednet.cn/Column.aspx?ColId=4");
//print_r($a->theindex());
?>